echo Hello\ World 
echo $PATH
#echo prints the content which is followed.
#\ means space. Bash is space sensitive and needs to be escaped.

which echo
#which prints that if i run echo,which echo will be run and show its path.

cd ~ 
#change pwd to user/home

cd / 
#change pwd to root

cd - 
#change pwd to the directory you were previously in

cd ../ 
#change pwd to previous level

pwd
#show your crruent working directory

ls 
#show the files and directories

ls -l
#show more details 

man ls
man cd
man ...
man signal.7
#show a manual page for us.Litte easy to read and navigate then --help

means clear

mv oldPath newPath
#rename or move

cp oldPath newPath
#copy

rm -r path 
rm path
#delete files. -r means recursion

rmdir path
#delete directories.

mkdir path
make 'my photo'
#make directory. you can use '' or "" means the whole string

sudo
# do as su

diff <(ls foo) <(ls bar)
#compare the difference between two files.

tldr ffmpeg

tldr elf

find . -name src -type d
find . -name foo -type f
find . -path '**/test/*.py' -type f

find . -name "*.tmp" -exec rm {} \;

locate "search_pattern"
#use index to fine files

updatedb
#update the database (index)

grep foobar mcd.sh
grep -R foobar .

history

history 10 | grep ffmpeg

echo hello > hello.txt
cat < hello.txt
cat < hello.txt > hello2.txt
cat cat < hello.txt >> hello2.txt
# >> means append

ls -l / | tail -n1 > ls.txt
#print the last line of "ls -l /"
#ls don't know tail and tail don't know ls

curl --head --silent google.com | grep -i content-length | cut --delimiter=' ' -f2

tree

autojump
j
#autojump - a faster way to navigate your filesystem
j opt
# jump to the opt directory you have gone before.
man autojump

echo 1060 | sudo tee brigthness
#Change the brightness of the device to 1060
#tee means write and show in shell

foo=bar
echo $foo
> bar
#foo = bar is wrong. Be careful with the space.

echo "Hello World"
>Hello World
#define a string

echo "Value is $foo"
> Value is bar
echo 'Value is $foo'
> Value is $foo
#the difference between "" and ''

foo=$(pwd)
#get the pwd's result,then give it to foo

cat <(ls) <(ls ..)
#() will make a tmp file to save the result

$0 
#name of program

$1,$2,$3
#args

$?
#get the error code from the previous command

$_
#get the last arg of the previous command

!!
#get the previous command

$$
#get previous command's pid

true
false

||
#and

&&
#or

;
#run it anyway

ls *.sh
#show the stuff which is endwith '.sh'

ls project?
#show the stuff which is project plus one any char.

touch foo{,1,2,10}
#creat foo  foo1  foo10  foo2 files

touch project{1,2}/test{1,2,3}
#creat 6 files with combination {} and {} 

mcd(){
 mkdir -p "$1"
 cd "$1"
}
#or 
source mcd.sh 

#!/bin/bash
#magic line

echo "Starting program at $(date)"

echo "Running program $0 with $# arguments with pid $$"

for file in "$@"; do
    grep foobar "$file" > /dev/null 2> /dev/null
    #redirect STDOUT and STDERR to a null register
    
    if [[ "$?" -ne 0 ]]; then
        echo "File $file does not have any foobar, adding one."
        echo "# foobar" >> "$file"
    fi
done

#!/usr/bin/env python
#use env to find python

import sys
for arg in reversed(sys.argv[1:]):
    print(arg)

./script.py a b c

def fizz_buzz(limit):
    for i in range(limit):
        if i % 3 == 0:
            print('fizz')
        if i % 5 == 0:
            print('fizz')
        if i % 3 and i % 5:
            print(i)

def main():
    fizz_buzz(10)

#use vim to fix it.   
import sys

def fizz_buzz(limit):
    for i in range(1,limit+1):
        if i % 3 == 0:
            print('fizz',end='')
        if i % 5 == 0:
            print('buzz',end='')
        if i % 3 and i % 5:
            print(i)

def main():
    fizz_buzz(int(sys.argv[1]))
    
if __name__ == '__main__':
    main()

journalctl 
#print log

less
#make pages

sed
#stream editor
#replace in the stream. Do it line by line. 
sed 's/.*Disconnected from//'
#use regex expression to replace
sed -E 's/(ab)*//g'
#use new regex rule
sed -E -n "s/.*Disconnected from user (.*) [0-9.]+ port [0-9]+/\1/p"
#get the secound capture gourp
# use -n and p command to only print the lines which were replaced.

wc 
wc -l
#print the num of lines

sort
sort -nk1,1
#-n means number sort
#-k means split by space of input
#1,1 start from first column and end at the first column, by this column
#type man sort or tdlr sort to know the args

uniq
uniq -c
#-c is count mode.

awk
awk '{print $2}'
#split by space and print the secound column
awk '$1 == 1 && $2 ~ /^c.*e$/ {print $0}'
#it will match 
#the first line of value is 1 and 
#the secound line which is start with c,end at e.
awk 'BEGIN {rows = 0} $1 == 1 && $2 ~ /^c.*e$/ {rows+=1} END {print rows}'

paste
paste -sd,
#receive a bunch of lines and pastes them together into a line
#use comma as delimiter

bc 
bc -l
#read the string and convert it to the arithmetic expression to calculate to result.

R
R --slave -e 'x <= scan(file="stdin",quiet=TRUE); summary(x)'
#statistics tool

gnuplot
gnuplot -p -e 'set boxwidth 0.5;plot "-" using 1:xtic(2) with boxes'
#Visual statistical tool 

xargs
cat log.txt | xargs echo
# xargs helps you use command which cannot receive a input and use args as input by |.
find /tmp -name core -type f -print | xargs /bin/rm -f
# Find files named core in or below the directory /tmp and delete them.  Note that this will work incorrectly if
find /tmp -name core -type f -print0 | xargs -0 /bin/rm -f
# Find  files named core in or below the directory /tmp and delete them, processing filenames in such a way that
cut -d: -f1 < /etc/passwd | sort | xargs echo
# Generates a compact listing of all the users on the system.

cat log.txt | grep sshd | sed -E -n "s/.*Disconnected from user (.*) [0-9.]+ port [0-9]+/\1/p" | sort | uniq -c | sort -nk1,1 |awk '{print $2}'|paste -sd,
# print sshd login users by ascending order

cat log.txt | grep sshd | sed -E -n "s/.*Disconnected from user (.*) [0-9.]+ port [0-9]+/\1/p" | sort | uniq -c | sort -nk1,1 |awk '{print $1}'|paste -sd+|bc -l
# print the sum of logins 

sleep
sleep 20

#!/usr/bin/env python
import signal,time

def handler(signum,time):
    print("\nI got a SIGINT, but I am not stopping")

signal.signal(signal.SIGINT,handler)
i = 0
while True:
    time.sleep(.1)
    print("\r{}".format(i), end="")
    i += 1

nohup sleep 2000 &
# run this program in the background.
# encapsulating the command you are executing
# ignore whatever signal you get, except KILL. 
# so that keep it running.

jobs
# print the programs which are running or suspended

bg %n
# n is a number. continue the suspended program n.
# send a continue signal

kill
# send any sort of Unix signal
kill -STOP %n
# send a STOP signal
kill -KILL %n
# send a KILL signal

tmux

tmux a -t foobar
# attach to a session called foobar

tmux new -t foobar
# create a new session called foobar

tmux ls
# print all sessions

alias ll="ls -lah"
alias mv="mv -i"

alias mv
> mv="mv -i"

alias mv='mv -i'

PS1="> "
# PS1 is a variable which control the prompt string of your bash. 

ln -s path/to/file symlink 

cd dotfiles
stow bash #make a symlink to .bashrc.
stow tmux # ...
stow vim
stow neovim

ssh jjgo@192.168.246.142
ssh jjgo@foobar.mit.edu
ssh jjgo@192.168.246.142 ls -la
ssh MINIPC
# execute the command on remote and send back the result.

#SSH Keys use keypair to verify you identification.
ssh-keygen -o -a 100 -t ed25519
# copy key in .ssh 

ssh-copy-id jjgo@192.168.246.142
# copy key to remote.

scp note.md jjgo@192.168.246.142:foobar.md
#copy more thing

rsync -avP . jjgo@192.168.246.142:cmd
# copy scores of thing. For example, use it to copy a folder.

Host MINIPC
    HostName IP
    User Sagiring
    Port 22
    IdentityFile path/to/privateKey


Host github.com
    User git
    Hostname github.com
    PreferredAuthentications publickey
    IdentityFile  path/to/privateKey

type blob = array<byte> #files
type tree = Map<String,tree | blob> #files or folders
type Commit = struct {
    parents:array<commit> #hash accutally
    author:Stirng
    message:String
    snapshot:tree #hash accutally
}
type object = blob | tree | commit

objects = Map<String,object>

def store(o):
    id = SHA1(o)
    objects[id] = o

def load(id):
    return objects[id]

git init
# git initialize

git help {Command}
git status
git add {files}
git commit (-a -m '')
git log --all --graph --decorate --oneline
#HEAD is point that point the commit where you actually are

git checkout {hash|name}
git checkout hello.txt #HEAD
# change your working directory files into {hash|name} status.
# move HEAD point into {hash}->content

git diff {hash} hello.txt
git diff {hash} HEAD hello.txt
# what changes from {hash} to HEAD
git diff hello.txt
# diff the HEAD and now status

git branch -vv
# more detail about branch
git branch cat
# create a new branch called cat
git checkout cat

git merge cat
#Auto-merging and merge conflict
git merge --abort
git add animal.py
git merge --continue
git mergetool

git remote
git remote add <name> <URL|ssh|filePath>
git remote add origin ../remote/


git push <remote> <local branch>:<remote branch>
git push origin master:master
# set git push by
git branch --set-upstream-to=origin/master


git clone <url> <folder>
git clone remote/ demo2

git fetch
#update from remote
git merge
git pull
# same with git fetch;git merge

git config
# vim ~/.gitconfig
git clone --shallow
# only clone the lastest snapshot.
git add -p animal.py
git diff --cached

git blame .bashrc
git show bd8d1510
#show more modification detail about file.

git stash
git stash pop
# here is a img example for use it.

git bisect
# auto find the problem commit

import logging
import sys

class CustomFormatter(logging.Formatter):
    """Logging Formatter to add colors and count warning / errors"""

    grey = "\x1b[38;21m"
    yellow = "\x1b[33;21m"
    red = "\x1b[31;21m"
    bold_red = "\x1b[31;1m"
    reset = "\x1b[0m"
    format = "%(asctime)s - %(name)s - %(levelname)s - %(message)s (%(filename)s:%(lineno)d)"

    FORMATS = {
        logging.DEBUG: grey + format + reset,
        logging.INFO: grey + format + reset,
        logging.WARNING: yellow + format + reset,
        logging.ERROR: red + format + reset,
        logging.CRITICAL: bold_red + format + reset
    }

    def format(self, record):
        log_fmt = self.FORMATS.get(record.levelno)
        formatter = logging.Formatter(log_fmt)
        return formatter.format(record)

# create logger with 'spam_application'
logger = logging.getLogger("Sample")

# create console handler with a higher log level
ch = logging.StreamHandler()
ch.setLevel(logging.DEBUG)

if len(sys.argv)> 1:
    if sys.argv[1] == 'log':
        ch.setFormatter(logging.Formatter('%(asctime)s : %(levelname)s : %(name)s : %(message)s'))
    elif sys.argv[1] == 'color':
        ch.setFormatter(CustomFormatter())

if len(sys.argv) > 2:
    logger.setLevel(logging.__getattribute__(sys.argv[2]))
else:
    logger.setLevel(logging.DEBUG)

logger.addHandler(ch)

# logger.debug("debug message")
# logger.info("info message")
# logger.warning("warning message")
# logger.error("error message")
# logger.critical("critical message")

import random
import time
for _ in range(100):
    i = random.randint(0, 10)
    if i <= 4:
        logger.info("Value is {} - Everything is fine".format(i))
    elif i <= 6:
        logger.warning("Value is {} - System is getting hot".format(i))
    elif i <= 8:
        logger.error("Value is {} - Dangerous region".format(i))
    else:
        logger.critical("Maximum value reached")
    time.sleep(0.3)

logger "Hello Logs"
journalctl --since "1m ago" | grep Hello

sudo strace -e ls -l > /dev/null
# print all syscalls that ls has executed.
# man strace to see more detail

# static analysis tools
pyflakes lint.py
mypy lint.py
writegoood note.md

import time, random
n = random.randint(1, 10) * 100

# Get current time
start = time.time()

# Do some work
print("Sleeping for {} ms".format(n))
time.sleep(n/1000)

# Compute time between start and now
print(time.time() - start)

# Output
# Sleeping for 500 ms
# 0.5713930130004883

time curl https://missing.csail.mit.edu &> /dev/null
# real    0m0.486s
# user    0m0.031s
# sys     0m0.001s

#!/usr/bin/env python

import sys, re

def grep(pattern, file):
    with open(file, 'r') as f:
        print(file)
        for i, line in enumerate(f.readlines()):
            pattern = re.compile(pattern)
            match = pattern.search(line)
            if match is not None:
                print("{}: {}".format(i, line), end="")

if __name__ == '__main__':
    times = int(sys.argv[1])
    pattern = sys.argv[2]
    for i in range(times):
        for file in sys.argv[3:]:
            grep(pattern, file)

python -m cProfile -s tottime grep.py 1000 '^(import|\s*def)[^,]*$' *.py

[omitted program output]

 ncalls  tottime  percall  cumtime  percall filename:lineno(function)
     8000    0.266    0.000    0.292    0.000 {built-in method io.open}
     8000    0.153    0.000    0.894    0.000 grep.py:5(grep)
    17000    0.101    0.000    0.101    0.000 {built-in method builtins.print}
     8000    0.100    0.000    0.129    0.000 {method 'readlines' of '_io._IOBase' objects}
    93000    0.097    0.000    0.111    0.000 re.py:286(_compile)
    93000    0.069    0.000    0.069    0.000 {method 'search' of '_sre.SRE_Pattern' objects}
    93000    0.030    0.000    0.141    0.000 re.py:231(compile)
    17000    0.019    0.000    0.029    0.000 codecs.py:318(decode)
        1    0.017    0.017    0.911    0.911 grep.py:3(<module>)

[omitted lines]

#!/usr/bin/env python
import requests
from bs4 import BeautifulSoup

# This is a decorator that tells line_profiler
# that we want to analyze this function
@profile
def get_urls():
    response = requests.get('https://missing.csail.mit.edu')
    s = BeautifulSoup(response.content, 'lxml')
    urls = []
    for url in s.find_all('a'):
        urls.append(url['href'])

if __name__ == '__main__':
    get_urls()

kernprof -l -v a.py

Total time: 0.636188 s
File: a.py
Function: get_urls at line 5

Line #  Hits         Time  Per Hit   % Time  Line Contents
==============================================================
 5                                           @profile
 6                                           def get_urls():
 7         1     613909.0 613909.0     96.5      response = requests.get('https://missing.csail.mit.edu')
 8         1      21559.0  21559.0      3.4      s = BeautifulSoup(response.content, 'lxml')
 9         1          2.0      2.0      0.0      urls = []
10        25        685.0     27.4      0.1      for url in s.find_all('a'):
11        24         33.0      1.4      0.0          urls.append(url['href'])

@profile
def my_func():
    a = [1] * (10 ** 6)
    b = [2] * (2 * 10 ** 7)
    del b
    return a

if __name__ == '__main__':
    my_func()

python -m memory_profiler example.py

Line #    Mem usage  Increment   Line Contents
==============================================
     3                           @profile
     4      5.97 MB    0.00 MB   def my_func():
     5     13.61 MB    7.64 MB       a = [1] * (10 ** 6)
     6    166.20 MB  152.59 MB       b = [2] * (2 * 10 ** 7)
     7     13.61 MB -152.59 MB       del b
     8     13.61 MB    0.00 MB       return a

sudo perf stat stress -c 1
sudo perf record stress -c 1
sudo perf report

htop

du -h
ncdu

lsof

hyperfine --warmup 3 'fd -e jpg' 'find . -iname "*.jpg"'

Benchmark #1: fd -e jpg
  Time (mean ± σ):      51.4 ms ±   2.9 ms    [User: 121.0 ms, System: 160.5 ms]
  Range (min … max):    44.2 ms …  60.1 ms    56 runs

Benchmark #2: find . -iname "*.jpg"
  Time (mean ± σ):      1.126 s ±  0.101 s    [User: 141.1 ms, System: 956.1 ms]
  Range (min … max):    0.975 s …  1.287 s    10 runs

Summary
  'fd -e jpg' ran
   21.89 ± 2.33 times faster than 'find . -iname "*.jpg"'

printf 'hello' | sha1sum